from urllib.request import urlopen
from bs4 import BeautifulSoup
import re

allInteralLinks = set()
allExteralLinks = set()
def getinternalLinks(bsObj, includeUrl):
    internalLinks = []
    for link in bsObj.findAll("a", href = re.compile("^(/|.*" + includeUrl + ")")):
        if link.attrs['href'] is not None:
            if link.attrs['href'] not in internalLinks:
                internalLinks.append(link.attrs['href'])
    return internalLinks

def getExternalLinks(bsObj, excludeUrl):
    externalLinks = []
    for link in bsObj.findAll("a", href = re.compile("^(http|www)((?!" + excludeUrl + ").)*$")):
        if link.attrs['href'] is not None:
            if link.attrs['href'] not in externalLinks:
                externalLinks.append(link.attrs['href'])
    return externalLinks

def splitAddress(address):
    addressParts = address.replace("http://", "").split("/")
    return addressParts

def getAllExteralLinks(siteUrl):
    html = urlopen(siteUrl)
    bsObj = BeautifulSoup(html, "html.parser")
    interalLinks = getinternalLinks(bsObj, splitAddress(siteUrl)[0])
    exteralLinks = getExternalLinks(bsObj, splitAddress(siteUrl)[0])
    for link in interalLinks:
        if link not in allInteralLinks:
            allInteralLinks.add(link)
            print(link)

    for link in exteralLinks:
        if link not in allExteralLinks:
            allExteralLinks.add(link)
            print(link)
            getAllExteralLinks(link)

getAllExteralLinks("https://fex.baidu.com")
